# -*- coding:utf-8 -*-

from scrapy import Request
from scrapyExercise.items import ShetuImageItem
from scrapy_redis.spiders import RedisSpider


class ShetuRedisSpider(RedisSpider):
    name = "sheturedis"
    redis_key = ""

    def parse(self, response, **kwargs):
        urls = response.xpath('//div[@class="pl-list"]/a[1]/@href').extract()
        # urls是相对地址，补全绝对地址
        url = response.urljoin(urls[0])
        yield Request(url, callback=self.parse_image)

    def parse_image(self, response, **kwargs):
        item = ShetuImageItem()
        urls = response.xpath('//li[@class="list"]/a/img/@data-original').extract()
        # 图片主题
        if urls:
            item['title'] = response.xpath('//li[@class="list"]/a/img/@title').extract_first()
            urls_ = []
            for u in urls:
                urls_.append(response.urljoin(u))
            item['image_urls'] = urls_
            yield item